{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "db5b4cbb",
   "metadata": {},
   "outputs": [],
   "source": [
    "# from huggingface_hub import snapshot_download\n",
    "\n",
    "# snapshot_download(\n",
    "#     repo_id=\"danavery/urbansound8K\",\n",
    "#     repo_type=\"dataset\",\n",
    "#     local_dir = './urbansound8K'\n",
    "# )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "a1a3d672",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6.6G\turbansound8K/data\r\n"
     ]
    }
   ],
   "source": [
    "!du -hs urbansound8K/data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "b17d35fd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data  README.md  UrbanSound8K.csv\r\n"
     ]
    }
   ],
   "source": [
    "!ls urbansound8K"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "d93feb5e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['urbansound8K/data/train-00005-of-00016-55ef1a0a51149c01.parquet',\n",
       " 'urbansound8K/data/train-00015-of-00016-03506887d89adfc9.parquet',\n",
       " 'urbansound8K/data/train-00009-of-00016-83fc7364d4798177.parquet',\n",
       " 'urbansound8K/data/train-00014-of-00016-c630762df85f6cf4.parquet',\n",
       " 'urbansound8K/data/train-00000-of-00016-e478d7cccca6a095.parquet',\n",
       " 'urbansound8K/data/train-00011-of-00016-79d186503a26671b.parquet',\n",
       " 'urbansound8K/data/train-00003-of-00016-691ee48aa53d9c1f.parquet',\n",
       " 'urbansound8K/data/train-00006-of-00016-0ef363072505e6d5.parquet',\n",
       " 'urbansound8K/data/train-00010-of-00016-4c1d0e285ed77843.parquet',\n",
       " 'urbansound8K/data/train-00004-of-00016-c0f37514d8e28a72.parquet',\n",
       " 'urbansound8K/data/train-00013-of-00016-17d827b1a5be0423.parquet',\n",
       " 'urbansound8K/data/train-00007-of-00016-dfac173beb21e5db.parquet',\n",
       " 'urbansound8K/data/train-00008-of-00016-2744487f32f65d77.parquet',\n",
       " 'urbansound8K/data/train-00012-of-00016-6aff88fdcca22918.parquet',\n",
       " 'urbansound8K/data/train-00002-of-00016-887e0748205b6fa9.parquet',\n",
       " 'urbansound8K/data/train-00001-of-00016-299138aa39afaa06.parquet']"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from tqdm import tqdm\n",
    "from glob import glob\n",
    "import os\n",
    "\n",
    "files = glob('urbansound8K/data/*.parquet')\n",
    "files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "5d19c9f9",
   "metadata": {},
   "outputs": [],
   "source": [
    "!mkdir urbansound8k"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "c8532225",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "urbansound8K/data/train-00005-of-00016-55ef1a0a51149c01.parquet\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████████| 546/546 [00:00<00:00, 1300.11it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "urbansound8K/data/train-00015-of-00016-03506887d89adfc9.parquet\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████████| 545/545 [00:00<00:00, 2104.43it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "urbansound8K/data/train-00009-of-00016-83fc7364d4798177.parquet\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████████| 546/546 [00:00<00:00, 1415.36it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "urbansound8K/data/train-00014-of-00016-c630762df85f6cf4.parquet\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████████| 545/545 [00:00<00:00, 1925.15it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "urbansound8K/data/train-00000-of-00016-e478d7cccca6a095.parquet\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████████| 546/546 [00:00<00:00, 1683.38it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "urbansound8K/data/train-00011-of-00016-79d186503a26671b.parquet\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████████| 546/546 [00:00<00:00, 2173.96it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "urbansound8K/data/train-00003-of-00016-691ee48aa53d9c1f.parquet\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████████| 546/546 [00:00<00:00, 1662.42it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "urbansound8K/data/train-00006-of-00016-0ef363072505e6d5.parquet\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████████| 546/546 [00:00<00:00, 1526.45it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "urbansound8K/data/train-00010-of-00016-4c1d0e285ed77843.parquet\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████████| 546/546 [00:00<00:00, 2019.18it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "urbansound8K/data/train-00004-of-00016-c0f37514d8e28a72.parquet\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████████| 546/546 [00:00<00:00, 1690.13it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "urbansound8K/data/train-00013-of-00016-17d827b1a5be0423.parquet\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████████| 545/545 [00:00<00:00, 2037.34it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "urbansound8K/data/train-00007-of-00016-dfac173beb21e5db.parquet\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████████| 546/546 [00:00<00:00, 1331.12it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "urbansound8K/data/train-00008-of-00016-2744487f32f65d77.parquet\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████████| 546/546 [00:00<00:00, 1521.19it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "urbansound8K/data/train-00012-of-00016-6aff88fdcca22918.parquet\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████████| 545/545 [00:00<00:00, 1926.47it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "urbansound8K/data/train-00002-of-00016-887e0748205b6fa9.parquet\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████████| 546/546 [00:00<00:00, 1860.21it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "urbansound8K/data/train-00001-of-00016-299138aa39afaa06.parquet\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████████| 546/546 [00:00<00:00, 1554.34it/s]\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "import random\n",
    "import json\n",
    "\n",
    "questions = [\n",
    "    'given the labels\\n{labels}\\n\\nwhat is the label for the audio',\n",
    "    'what is the label for audio\\n\\nthe labels: {labels}'\n",
    "]\n",
    "\n",
    "train = []\n",
    "\n",
    "for f in files:\n",
    "    print(f)\n",
    "    df = pd.read_parquet(f)\n",
    "    for i in tqdm(range(len(df))):\n",
    "        metadata = df.iloc[i].to_dict()\n",
    "        metadata.pop('audio')\n",
    "        audio_filename = os.path.join('urbansound8k', df['slice_file_name'].iloc[i])\n",
    "        with open(audio_filename, 'wb') as fopen:\n",
    "            fopen.write(df['audio'].iloc[i]['bytes'])\n",
    "        train.append({\n",
    "            'question': random.choice(questions).format(labels = unique_labels),\n",
    "            'answer': df['class'].iloc[i],\n",
    "            'audio_filename': audio_filename,\n",
    "            'metadata': json.dumps(metadata)\n",
    "        })"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "0ddc5b0b",
   "metadata": {},
   "outputs": [],
   "source": [
    "from datasets import Dataset\n",
    "\n",
    "dataset = Dataset.from_list(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "40b075ce",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "96258ae31ff24bc5ad5ef8095d32d47a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Uploading the dataset shards:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1f3fcc7a0b19411988d02907b21b5d61",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Creating parquet from Arrow format:   0%|          | 0/9 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "52d07300f96f4ddf9f8a5cc89645b552",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Uploading...:   0%|          | 0.00/330k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7740c8963fb6429784194a9799c7389e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "README.md:   0%|          | 0.00/2.03k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "CommitInfo(commit_url='https://huggingface.co/datasets/mesolitica/Zeroshot-Audio-Classification-Instructions/commit/65f34137bb2e3106d7c0d9074bda4aaee36a772e', commit_message='Upload dataset', commit_description='', oid='65f34137bb2e3106d7c0d9074bda4aaee36a772e', pr_url=None, repo_url=RepoUrl('https://huggingface.co/datasets/mesolitica/Zeroshot-Audio-Classification-Instructions', endpoint='https://huggingface.co', repo_type='dataset', repo_id='mesolitica/Zeroshot-Audio-Classification-Instructions'), pr_revision=None, pr_num=None)"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset.push_to_hub('mesolitica/Zeroshot-Audio-Classification-Instructions', split = 'urbansound8k')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "2f4afb82",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8732"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from glob import glob\n",
    "\n",
    "audio_files = glob('urbansound8k/*.wav')\n",
    "len(audio_files)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "1254a797",
   "metadata": {},
   "outputs": [],
   "source": [
    "import zipfile\n",
    "with zipfile.ZipFile('urbansound8k.zip', 'w', zipfile.ZIP_DEFLATED) as zipf:\n",
    "    for f in audio_files:\n",
    "        zipf.write(f, arcname=f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "50d30063",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Uploading files using Xet Storage..\n",
      "Uploading...: 100%|████████████████████████| 6.02G/6.02G [01:26<00:00, 70.0MB/s]\n",
      "https://huggingface.co/datasets/mesolitica/Zeroshot-Audio-Classification-Instructions/blob/main/urbansound8k.zip\n"
     ]
    }
   ],
   "source": [
    "!huggingface-cli upload mesolitica/Zeroshot-Audio-Classification-Instructions urbansound8k.zip \\\n",
    "--repo-type=dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "ea66dd06",
   "metadata": {},
   "outputs": [],
   "source": [
    "!rm urbansound8k.zip"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
